Preprocessing Part

import json 
import pandas as pd
import pymorphy2
import string
from nltk import ngrams
import re
import spacy

morph = pymorphy2.MorphAnalyzer() #Yet unused pymrphy2, very capable for russian texts (POS, word form editing etc)
nlp = spacy.load("ru_core_news_lg")
special_symbols = ['\u200b', '\n'] #emojis, unicode characters,etc
#A bigger set of custom stopwords. Domain ambiguous.
#works fine everywhere but you may still want to check if it includes a very specific word for your task
custom_stopwords = json.load(open('stopwords-ru.json', encoding='utf-8-sig'))  # load custom stopwords
nlp.Defaults.stop_words = nlp.Defaults.stop_words.union(custom_stopwords) #update spacy stopwords

def splitter(text):
    """A function to split text by '.', '!' or '?', I also added ';' to break ideas within text."""
    return [a for a in re.split(r'[.?!;]\s*', text) if len(a)>=10]


def n_grams(words,n_grams = [2]):
    """
    A wrapping function for nltk's ngram function
    """
    ng = []
    for length in set(n_grams):
        grams = ngrams(words, length)
        ng.extend([' '.join(gram) for gram in grams])
    return ng

def clean_text(text):
    """
    Main text cleaning function. Removing unneeded symbols, whitespaces, etc
    Also removing stop-words
    """
    text = re.sub('http[s]?://\S+', '', text)  # remove the links
    for sym in special_symbols:
        text = text.replace(sym, '')
    text = re.sub(r'[^\w\s]', ' ', text.lower())  
    text = re.sub(' +', ' ', text)  # remove unnecessary spaces
    # remove stopwords return final string
    return ' '.join([token.text for token in nlp(text) if not nlp.vocab[token.lemma_].is_stop])


def split_text(text):
    """
    A very simple function to split clean texts into list of words
    """
    ordered_word_list = text.split()
    return ordered_word_list


def rm_digit(text):
    """
    Additional function to remove any digit from the text
    """
    return re.sub(r'[0-9]+', '', text)


def process(text):
    """
    Processing pipeline with all aforementioned steps, except for digit removal
    """
    #Break text in sentences, remove stopwords etc
    texts = splitter(text)
    res = []
    for txt in texts:
        txt = clean_text(txt)
        words = split_text(txt)
        res.append(words)
    return res

def process_drop_digit(text):
    """
    Processing pipeline with additional digit removal
    """
    texts = splitter(text)
    res = []
    for txt in texts:
        txt = clean_text(txt)
        txt = rm_digit(txt)
        words = split_text(txt)
        res.append(words)
    return res


def process2(wordlists):
    """
    A second part of the pipeline. From all words in text select specific POS and lemmatize what is needed
    """
    #Use broken texts to get lemmas of each word and remove certain parts of speech
    res = {}
    lemmas = []
    nadj_lemmas = []
    vnadv_lemmas = []
    nouns_only = []
    #ADV NOUN VERB 'ADJ'
    for sentence_words in wordlists:
        sen_lemma = []
        sen_nadj_lemma = []
        sen_vnadv_lemma = []
        for token in nlp(' '.join(sentence_words)):
            lemma = token.lemma_
            pos = token.pos_
            sen_lemma.append(lemma)
            if pos == 'NOUN':
                nouns_only.append(lemma)
                sen_nadj_lemma.append(lemma)
                sen_vnadv_lemma.append(lemma)
            elif pos == 'ADJ':
                sen_nadj_lemma.append(lemma)
            elif pos in ['ADV', 'VERB']:
                sen_vnadv_lemma.append(lemma)   
        if sen_lemma !=[]:
            lemmas.append(sen_lemma)
        if sen_nadj_lemma !=[]:
            nadj_lemmas.append(sen_nadj_lemma)
        if sen_vnadv_lemma !=[]:
            vnadv_lemmas.append(sen_vnadv_lemma)
    return {'nouns':list(set(nouns_only)),'lemmas':lemmas, 'nadj_lemmas': nadj_lemmas, 'vnadv_lemmas':vnadv_lemmas}
            

### This part of code is here to speed up the processing part with multithreading
from multiprocess import Pool
import numpy as np
import datetime

def batch_calc(df):
    #Batch calc, something that happens in parallel with multiple processes
    indexes = df.index.values
    result = {}
    for idx in indexes:
        result.update({idx: process(df[df.index == idx].Text.values[0])})
    return result

def batch_calc_digit(df):
    #Batch calc, something that happens in parallel with multiple processes
    indexes = df.index.values
    result = {}
    for idx in indexes:
        result.update({idx: process_drop_digit(df[df.index == idx].Text.values[0])})
    return result

def batch_calc2(df):
    #Batch calc, something that happens in parallel with multiple processes
    indexes = df.index.values
    result = {}
    for idx in indexes:
        result.update({idx: process2(df[df.index == idx].broken_text.values[0])})
    return result

def batch_calc3(df):
    #Batch calc NER
    indexes = df.index.values
    result = {}
    for idx in indexes:
        result.update({idx: [str(a) for a in nlp(df[df.index == idx].ltext.values[0]).ents]})
    return result

def pcc(df,func = batch_calc, n_cores = 16):
    #Main parallel calc function
    results = {}
    pool = Pool(n_cores)
    split = np.array_split(df.index.values, n_cores)
    split_dfs = [df[df.index.isin(idx)] for idx in split]
    res = pool.map(func, split_dfs)
    for i in res:
        if i != {}:
            results.update(i)
    pool.close()
    pool.join()
    return results

### Main Calculation part. Loading initial Json data and forming a dataframe from it
with open("220408-sentencized-primer.jsonl", 'r') as f:
    di = f.read()
    dates = []
    ids = []
    texts = []
    titles = []
    for string in di.split('\n'):
        dictionary = json.loads(string.encode('utf-8'))
        dates.append(dictionary['Date'])
        texts.append(dictionary['Text'])
        titles.append(dictionary['Title'])
        ids.append(dictionary['ID'])
    df = pd.DataFrame({'id' : ids, 'Title': titles, 'Date': dates, 'Text' : texts})
    df = df.set_index('id') 
    ndf = df[['Text','Title','Date']].groupby(['Title','Date']).agg(lambda x: ''.join(x))
    ndf = ndf.reset_index()
    brokens = pcc(ndf,func = batch_calc_digit)
    ndf['broken_text'] = [brokens[id] for id in ndf.index]
    lemma_texts = pcc(ndf, func = batch_calc2)
    ndf['nouns'] = [lemma_texts[id]['nouns'] for id in ndf.index]
    ndf['lemmas'] = [lemma_texts[id]['lemmas'] for id in ndf.index]
    ndf['nadj_lemmas'] = [lemma_texts[id]['nadj_lemmas'] for id in ndf.index]
    ndf['vnadv_lemmas'] = [lemma_texts[id]['vnadv_lemmas'] for id in ndf.index]
    ndf.to_csv('texts_no_num.csv', index = False)

BERTopic modeling

import pandas as pd
df = pd.read_csv('texts_no_num.csv', converters = {'nouns':eval,'broken_text': eval, 'lemmas':eval, 'vnadv_lemmas' : eval})
df.Date = pd.to_datetime(df.Date)
df['ltext'] = df.lemmas.apply(lambda ar: ' '.join([' '.join(sr) for sr in ar]))

from bertopic import BERTopic
from hdbscan import HDBSCAN

hdbscan_model = HDBSCAN(metric='euclidean', 
                        cluster_selection_method='eom', prediction_data=True,
                        min_samples=10, min_cluster_size = 20)
model = BERTopic(nr_topics='auto', hdbscan_model=hdbscan_model, language="russian",
                       calculate_probabilities=True, low_memory=True, verbose=True)
topics, probs = model.fit_transform(df.ltext.tolist())

Batches:   0%|          | 0/358 [00:00<?, ?it/s]

2022-05-23 14:45:34,433 - BERTopic - Transformed documents to Embeddings
2022-05-23 14:45:46,974 - BERTopic - Reduced dimensionality with UMAP
2022-05-23 14:45:50,634 - BERTopic - Clustered UMAP embeddings with HDBSCAN
2022-05-23 14:46:21,955 - BERTopic - Reduced number of topics from 96 to 25

model.save('best_model')

model = BERTopic().load('best_model')

df['prob'] = [prob for prob in probs]
df['topic'] = topics

model.get_topic_freq()

	Topic	Count
0	0	4927
1	-1	4705
2	1	374
3	2	156
4	3	125
5	4	121
6	5	103
7	6	100
8	7	95
9	8	92
10	9	89
11	10	78
12	11	69
13	12	54
14	13	49
15	14	39
16	15	37
17	16	35
18	17	35
19	18	30
20	19	29
21	20	28
22	21	26
23	22	22
24	23	21

model.visualize_topics()

model.visualize_barchart(top_n_topics=20, n_words=10, height = 500, width = 300)

#topics_over_time
topics_over_time = model.topics_over_time(df.ltext, df.topic, df.Date.apply(lambda dt: dt.replace(day=1).replace(month=1)))
model.visualize_topics_over_time(topics_over_time,top_n_topics=20,normalize_frequency=False)

Additional Clustering Metrics

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.metrics import silhouette_samples, silhouette_score

def get_silhouette(texts, labels):
    """
    TF-IDF based silhouette calculation.
    Note that another tokenization approach was used in BERTtopic
    """
    tfidf_vectorizer = TfidfVectorizer(use_idf=True)
    X = tfidf_vectorizer.fit_transform(texts).todense()
    data2D = PCA(n_components=2).fit_transform(X)
    return silhouette_samples(data2D, labels)

df['silhouette'] = get_silhouette(df.ltext, df.topic)

print(df['silhouette'].mean())

-0.4761763599291887

import gensim.corpora as corpora
from gensim.models.coherencemodel import CoherenceModel

# Preprocess documents
cleaned_docs = model._preprocess_text(df.ltext.tolist())

# Extract vectorizer and tokenizer from BERTopic
vectorizer = model.vectorizer_model
tokenizer = vectorizer.build_tokenizer()

# Extract features for Topic Coherence evaluation
words = vectorizer.get_feature_names()
tokens = [tokenizer(doc) for doc in cleaned_docs]
dictionary = corpora.Dictionary(tokens)
corpus = [dictionary.doc2bow(token) for token in tokens]
topic_words = [[words for words, _ in model.get_topic(topic)] 
               for topic in range(len(set(topics))-1)]

# Evaluate
coherence_model = CoherenceModel(topics=topic_words, 
                                 texts=tokens, 
                                 corpus=corpus,
                                 dictionary=dictionary, 
                                 coherence='c_v')
coherence = coherence_model.get_coherence()

coherence

0.6948253840339905

import seaborn as sns
import matplotlib.pyplot as plt
coherence_per_topic = coherence_model.get_coherence_per_topic()
topics_str = [ '\n '.join(str(t)) for t in range(len(topic_words))]
data_topic_score = pd.DataFrame( data=zip(topics_str, coherence_per_topic), columns=['Topic', 'Coherence'] )
data_topic_score = data_topic_score.set_index('Topic')

fig, ax = plt.subplots( figsize=(7,16) )
ax.set_title("Topics coherence\n $C_v$")
sns.heatmap(data=data_topic_score, annot=True, square=True,
            cmap='Reds', fmt='.2f',
            linecolor='black', ax=ax )
plt.yticks( rotation=0 )
ax.set_xlabel('')
ax.set_ylabel('')
fig.show()

### Overall topic metrics
x = df[['topic', 'Date']].groupby(['topic']).mean()
x = x.reset_index()
x['key'] = x['topic'].apply(lambda topic_id: '_'.join([w[0] for w in find_unique_grams(df, topic_id, ngram_range = [1], n = 5)]))
x['silhouette'] = x['topic'].apply(lambda topic_id: df[df.topic == topic_id].silhouette.mean())
x['avg_len'] = x['topic'].apply(lambda topic_id: df[df.topic == topic_id].Text.apply(len).mean())

x

	topic	Date	key	silhouette	avg_len
0	-1	2011-06-19 20:45:20.850159360	федеральный	-0.523334	13394.403613
1	0	2011-01-11 10:54:58.234219520	государственный_экономика	-0.529835	12870.968947
2	1	2012-12-26 20:08:59.037433088	спорт_спортивный_олимпийский_игра_подготовка	-0.384719	11523.721925
3	2	2012-02-17 23:41:32.307692288	господин_израиль_сотрудничество_международный_...	-0.150610	4505.737179
4	3	2009-07-06 23:25:26.400000000	суд_право_орган_гражданин_закон	-0.037142	10110.296000
5	4	2013-08-19 02:22:48.595041280	газ_рынок_компания_миллиард_энергетический	-0.325993	12140.752066
6	5	2012-03-04 01:37:51.844660224	банк_финансовый_банковский_кредит_рынок	-0.050580	5500.961165
7	6	2010-08-13 06:43:12.000000000	сотрудничество_господин_международный_бразилия...	-0.198663	7081.820000
8	7	2011-05-05 07:19:34.736841984	церковь_православный_патриарх_духовный_религио...	-0.001458	5793.568421
9	8	2013-04-24 01:33:54.782608640	транспортный_транспорт_строительство_перевозка...	-0.291644	16409.043478
10	9	2020-05-17 08:05:23.595505664	тысяча_медицинский_мера_рубль_поддержка	-0.204372	24419.224719
11	10	2009-11-17 19:41:32.307692288	индия_сотрудничество_индийский_область_господин	-0.174626	7521.974359
12	11	2014-05-12 14:36:31.304347904	сирия_сирийский_господин_процесс_международный	-0.310198	9071.971014
13	12	2014-06-23 10:40:00.000000000	африканский_сотрудничество_африка_господин_меж...	-0.140676	5101.351852
14	13	2012-12-21 00:29:23.265306112	медицинский_здравоохранение_помощь_центр_врач	-0.218508	19922.306122
15	14	2008-05-21 01:50:46.153846272	пенсионный_пенсия_рубль_принять_гражданин	-0.077648	10759.923077
16	15	2007-04-29 07:08:06.486486528	болгария_болгарский_господин_сотрудничество_эн...	-0.287147	6329.513514
17	16	2011-04-10 01:22:17.142857216	вьетнам_сотрудничество_вьетнамский_господин_визит	-0.189481	5600.400000
18	17	2013-01-30 02:03:25.714285824	сербия_господин_сотрудничество_сербский_междун...	-0.107974	6240.857143
19	18	2013-10-31 05:36:00.000000000	морской_флот_военно_корабль_подводный	0.174094	3491.300000
20	19	2012-05-15 06:37:14.482758656	космический_космос_деятельность_результат_задача	-0.228937	8892.172414
21	20	2009-09-21 17:08:34.285714176	греция_сотрудничество_греческий_господин_между...	-0.271556	7053.357143
22	21	2008-12-31 00:00:00.000000000	господин_министр_сотрудничество_малайзия_премьер	-0.148072	3305.807692
23	22	2008-09-07 06:32:43.636363520	молдова_сотрудничество_молдавский_молдавия_рес...	0.036282	5086.818182
24	23	2012-05-22 17:08:34.285714176	корея_корейский_сотрудничество_республика_госп...	-0.144353	5372.523810

Word Filters

df = pd.read_csv('df_all.csv', converters = {'nouns':eval,'broken_text': eval, 'lemmas':eval, 'vnadv_lemmas' : eval})

from collections import Counter

def cnvrt(top_grams):
    """
    Simple function to convert tuples with frequencies to simple sets of words
    """
    return set([w[0] for w in top_grams])

def find_top_grams(clean_texts, ngram_range = [1], n= 5, setify = False):
    """
    Function that looks for a top freqent n-grams in group of texts directly
    clean_texts - expected input of texts in form of [[sentence1words],[sentence2words]...]
    ngram_range - can modify filter to look for bigrams instead of unigrams etc
    n - how many n-grams do you expect in output
    """
    res = []
    for gram in ngram_range:
        if gram == 1:
            res.append([word for text in clean_texts for sent in text for word in sent ])
        else:
            #[word for text in clean_texts for sent in text for word in sent ]
            res.append([' '.join(t) for text in clean_texts for sent in text for t in (ngrams(sent,gram))])
    res = [item for subitm in res for item in subitm]
    occurence_count = Counter(res)
    if n!=0:
        if not setify:
            return occurence_count.most_common(n)
        else:
            return cnvrt(occurence_count.most_common(n))
    else:
        if not setify:
            return occurence_count.most_common(len(set(res)))
        else:
            return cnvrt(occurence_count.most_common(len(set(res))))

find_top_grams(df[df.topic == 3].lemmas, ngram_range = [3], n = 5, setify = True)

{'военный технический сотрудничество',
 'вооружение военный техника',
 'добрый уважаемый коллега',
 'оборонно промышленный комплекс',
 'путин добрый уважаемый'}

find_top_grams(df[df.topic == 12].lemmas, ngram_range = [3], n = 10)

[('субъект российский федерация', 90),
 ('федеральный целевой программа', 56),
 ('путин добрый уважаемый', 46),
 ('добрый уважаемый коллега', 46),
 ('уважаемый владимир владимирович', 44),
 ('правительство российский федерация', 41),
 ('депутат государственный дума', 33),
 ('эффективность бюджетный расход', 26),
 ('повышение заработный плата', 26),
 ('налог добавить стоимость', 23)]

from matplotlib import pyplot as plt
from matplotlib_venn import venn2, venn3
from matplotlib_venn import venn2_unweighted,venn3_unweighted

def venn_topics(topic_top_grams):
    """
    It is possible to visualize those top n-grams on venn diagram
    Potential use case could include comparing different topics or different dates, information sources, etc.
    """
    if len(topic_top_grams) == 2:
        A = topic_top_grams[0]
        B = topic_top_grams[1]
        diagram = venn2_unweighted([A,B], ("topic A", "topic B"))
        diagram.get_label_by_id("10").set_text("\n".join(A - B))
        diagram.get_label_by_id("11").set_text("\n".join(A & B))
        diagram.get_label_by_id("01").set_text("\n".join(B - A))
        plt.gcf().canvas.set_window_title('Fun With Venn Diagrams')  # Set window title
        plt.gcf().set_size_inches(15,15)
        plt.show()
    if len(topic_top_grams) == 3:
        A = topic_top_grams[0]
        B = topic_top_grams[1]
        C = topic_top_grams[2]
        diagram = venn3_unweighted([A,B,C], ("topic A", "topic B","topic C"))
        diagram.get_label_by_id("100").set_text("\n".join(A - B - C))
        diagram.get_label_by_id("010").set_text("\n".join(B - A - C))
        diagram.get_label_by_id("001").set_text("\n".join(C - B - A))
        diagram.get_label_by_id("110").set_text("\n".join(A & B - C))
        diagram.get_label_by_id("101").set_text("\n".join(A & C - B))
        diagram.get_label_by_id("011").set_text("\n".join(C & B - A))
        diagram.get_label_by_id("111").set_text("\n".join(A & B & C))
        plt.gcf().canvas.set_window_title('Fun With Venn Diagrams')  # Set window title
        plt.gcf().set_size_inches(15,15)
        plt.show()

venn_topics([find_top_grams(df[df.topic == 12].lemmas, ngram_range = [2], n = 15, setify = True),
             find_top_grams(df[df.topic == 11].lemmas, ngram_range = [2], n = 15, setify = True),
             find_top_grams(df[df.topic == 5].lemmas, ngram_range = [2], n = 15, setify = True)])

def find_unique_grams(df, topic_id,ngram_range = [2], n = 5):
    """
    This function substracts top n-grams of all texts not belonging to a topic from top n-grams within this topic,
    thus leaving you with top unique n-grams within selected topic
    """
    main = find_top_grams(df[df.topic == topic_id].lemmas, ngram_range = ngram_range, n = 5*n)
    rest = find_top_grams(df[df.topic != topic_id].lemmas, ngram_range = ngram_range, n = 5*n)
    rest = [w[0] for w in rest]
    return [w for w in main if w[0] not in rest][:n]

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(stop_words=nlp.Defaults.stop_words, ngram_range = (2, 2))
X = tfidf.fit_transform(df.ltext)
feature_names = np.array(tfidf.get_feature_names())


def get_top_tf_idf_words(docs, top_n=2):
    """
    Alternative way of finding top words based to TF-IDF transform
    Note that ngram range has to be specified beforehand when creating an instance of TfidfVectorizer
    """
    responses = tfidf.transform(docs)
    res = []
    for response in responses:
        sorted_nzs = np.argsort(response.data)[:-(top_n+1):-1]
        res.append(list(feature_names[response.indices[sorted_nzs]]))
    return res

### Top 5 bigrams for every text within dataset
df['tf_idf_top_5'] = get_top_tf_idf_words(df['ltext'], top_n = 5)

### Basic NER, function definition took place in preprocessing part due to the multithreaded approach
ents = pcc(df,func = batch_calc3, n_cores = 16)
df['ents'] = [ents[id] for id in df.index]

from collections import Counter

def top_ents(df_group, top_n = 10):
    """
    Showing top entitie for a df_group subset
    """
    ents = [ent for text_ents in df_group.ents for ent in text_ents]
    occurence_count = Counter(ents)
    return occurence_count.most_common(top_n)

top_ents(df[df.topic == 10], top_n = 20)

[('сирия', 705),
 ('путин', 634),
 ('израиль', 429),
 ('иран', 331),
 ('египет', 252),
 ('ближний восток', 205),
 ('оон', 159),
 ('турция', 158),
 ('саудовский аравия', 144),
 ('ирак', 133),
 ('сша', 78),
 ('сочи', 75),
 ('владимир путин', 62),
 ('медведев', 61),
 ('ливан', 60),
 ('иордания', 59),
 ('рухани', 50),
 ('эрдоган', 50),
 ('аббас', 39),
 ('нетаньяху', 33)]

top_ents(df[df.topic == 3], top_n = 20)

[('путин', 1331),
 ('владимир владимирович', 420),
 ('медведев', 293),
 ('опк', 292),
 ('нато', 175),
 ('снг', 168),
 ('минобороны', 160),
 ('сирия', 147),
 ('одкб', 117),
 ('афганистан', 115),
 ('роскосмос', 102),
 ('сша', 98),
 ('европа', 97),
 ('фсб', 93),
 ('оон', 89),
 ('китай', 73),
 ('дальний восток', 64),
 ('втс', 63),
 ('дмитрий анатолиевич', 61),
 ('америка', 61)]

def keyword_filter_tfidf(df, keyword, sent_range = 3,n_gram_r = [2], top_n = 10):
    """
    An approach towards counting top ngrams within certain segment of the text
    (In range of sent_range sentences from a sentence where keyword word/ngram was found)
    """
    gram = len(keyword.split(' '))
    res = []
    for ind,r in df.iterrows():
        for i,sent in enumerate(r.lemmas):
            grams = n_grams(sent,[gram])
            if keyword in grams:
                sents = r.lemmas[max(0, i-sent_range + 1) : min(i+sent_range, len(r.lemmas) - 1)]
                #res.append(n_grams(sents, [gram]))
                res.append([n_grams(sent, n_gram_r) for sent in sents])
                break
    res = [g for sent_grams in res for grams in sent_grams for g in grams]
    counts = Counter(res).most_common(top_n)
    return [w for w in counts if w[0]!=keyword]

keyword_filter_tfidf(df,"рамзан", sent_range = 5,n_gram_r = [3], top_n = 20)

[('путин рамзан ахматович', 13),
 ('кадыров владимир владимирович', 9),
 ('республика рамзан кадыров', 6),
 ('рамзан ахматович кадыров', 6),
 ('президент чеченский республика', 6),
 ('рамзан ахматович поговорить', 6),
 ('чеченский республика рамзан', 5),
 ('глава чеченский республика', 4),
 ('чечня рамзан кадыров', 4),
 ('ахмат хаджи кадыров', 4),
 ('правительство чеченский республика', 3),
 ('территория чеченский республика', 3),
 ('республика рамзан ахматович', 3),
 ('глава республика рамзан', 3),
 ('чеченский республика герой', 2),
 ('уважаемый владимир владимирович', 2),
 ('социально экономический развитие', 2),
 ('глава чечня рамзан', 2),
 ('линия сельский хозяйство', 2),
 ('ход избирательный кампания', 2)]

keyword_filter_tfidf(df,"украина", sent_range = 10)

[('российский федерация', 355),
 ('владимир владимирович', 325),
 ('президент украина', 159),
 ('уважаемый коллега', 135),
 ('господин президент', 134),
 ('точка зрение', 107),
 ('дмитрий анатолиевич', 87),
 ('экономический пространство', 87),
 ('единый экономический', 82),
 ('путин уважаемый', 82)]

### A try to show how presence of a certain keyword changes over time
### Idea is simple: check if keyword is present in the text for every text and plot this development over time
def grams_over_time(texts, keyword):
    gram = len(keyword.split(" "))
    grams = n_grams(texts.split(" "), [gram])
    return int(keyword in grams)

df['check'] = df['ltext'].apply(lambda text : grams_over_time(text, "экономический пространство"))
df['Date'] = pd.to_datetime(df['Date'])
df[['Date','check']].resample('W-Mon', on='Date').sum().plot()

<AxesSubplot:xlabel='Date'>

df['check'] = df['ltext'].apply(lambda text : grams_over_time(text, "украина"))
df['Date'] = pd.to_datetime(df['Date'])
df[['Date','check']].resample('W-Mon', on='Date').sum().plot()

<AxesSubplot:xlabel='Date'>

df.to_csv('df_all.csv')

df.head()

	Unnamed: 0	Title	Date	Text	broken_text	nouns	lemmas	nadj_lemmas	vnadv_lemmas	ltext	topic	silhouette	ents
0	0	1 марта вступил в силу закон «О полиции»	2011-03-01	* * * Д.Медведев: Сегодня вступает в силу зако...	[[медведев, вступает, закон, полиции], [надеюс...	[январь, свобода, вступление, начальник, право...	[[медведев, вступать, закон, полиция], [надеят...	[['закон', 'полиция'], ['служба', 'вступление'...	[[вступать, закон, полиция], [надеяться, служб...	медведев вступать закон полиция надеяться служ...	5	-0.387181	[медведев, мвд, сергей нарышкин, мвд, нургалие...
1	1	10-летие вещания Russia Today	2015-12-10	Глава государства, в частности, осмотрел инста...	[[глава, государства, частности, осмотрел, инс...	[залог, трибуна, аудитория, политик, право, мн...	[[глава, государство, частность, осмотреть, ин...	[['глава', 'государство', 'частность', 'инстал...	[[глава, государство, частность, осмотреть, ин...	глава государство частность осмотреть инсталля...	0	-0.251035	[russia today, russia today, путин, russia tod...
2	2	100-летие русской часовни у перевала Вршич	2016-07-30	В июле 1915 года вблизи города Краньска-Гора б...	[[июле, вблизи, краньска, гора, организован, л...	[выбор, подвиг, лавина, общение, любовь, страд...	[[июле, вблизи, краньска, гора, организовать, ...	[['июле', 'гора', 'лагерь', 'военнопленный', '...	[[июле, гора, организовать, лагерь, военноплен...	июле вблизи краньска гора организовать лагерь ...	0	-0.131249	[краньска, вршич, владимир путин, пахор, слове...
3	3	12 июля в России – День траура по погибшим в р...	2011-07-11	Дмитрий Медведев выразил соболезнования родным...	[[дмитрий, медведев, выразил, соболезнования, ...	[транспорт, прокурор, поиску, соболезнование, ...	[[дмитрий, медведев, выразить, соболезнование,...	[['соболезнование', 'родный', 'близким'], ['уч...	[[выразить, соболезнование, близким, погибнуть...	дмитрий медведев выразить соболезнование родны...	-1	-0.526531	[дмитрий медведев, мчс, сергей шойгу, игорь ле...
4	4	26 декабря в Российской Федерации объявлено дн...	2016-12-25	В.Путин: Прежде всего хочу выразить самые искр...	[[путин, прежде, выразить, самые, искренние, с...	[транспорт, причина, траур, связь, море, собол...	[[путин, прежде, выразить, самые, искренний, с...	[['самые', 'искренний', 'соболезнование', 'сем...	[[прежде, выразить, соболезнование, семья, гра...	путин прежде выразить самые искренний соболезн...	17	-0.296759	[путин, назарбаев, владимир владимирович, каза...